package com.laivi.basic.module.crawler;

import java.util.concurrent.LinkedBlockingQueue;

import com.laivi.basic.model.crawler.BloomFilter;
import com.laivi.basic.model.crawler.LinkParser;

/**
 * @author laivi.zhu@gmail.com
 * @date 2014-6-16 description:
 */
public class CrawlerUrl implements Runnable {

	private LinkedBlockingQueue<String> parsereddQueue;
	private LinkedBlockingQueue<String> parseringQueue;
	private BloomFilter filter;

	/**
	 * @param queue
	 */
	public CrawlerUrl(LinkedBlockingQueue<String> parsereddQueue,
			LinkedBlockingQueue<String> parseringQueue, BloomFilter filter) {
		super();
		this.parsereddQueue = parsereddQueue;
		this.parseringQueue = parseringQueue;
		this.filter = filter;
	}

	@Override
	public void run() {
		while (true) {
			try {
				String url = parseringQueue.take();
				System.out.println("paser remove Url:"+url);
				System.out.println("parseringfilter Size:"+filter.getSize());
				if (!filter.contains(url)) {
					filter.add(url);
					for (String seed : LinkParser.extractLinks(url)) {
						System.out.println("paser add Url:"+seed);
						parseringQueue.put(seed);
					}
				}
				parsereddQueue.put(url);
				System.out.println("pasered add Url:"+url);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
	}

}
